import requests
import json
from dbutil.pgsql import PgsqlPipeline
from datetime import date

def get_provinces():
	url = "http://www.dfyl-luxgen.com/Index/api/area"
	r = requests.get(url)
	provinces = json.loads(r.text)["data"]
	return provinces

def get_cities(cid):
	param = {"cid": cid}
	url = "http://www.dfyl-luxgen.com/Index/api/area"
	r = requests.post(url, data=param)
	cities = json.loads(r.text)["data"]
	return cities

def get_dealers(pid, cid):
	param = {
			"province": pid,
			"city": cid,
			"is_dealer": "",
			"dealername": ""
			}
	url = "http://www.dfyl-luxgen.com/Index/api/search"
	r = requests.post(url, data=param)
	dealers = json.loads(r.text)["data"]
	return dealers

def get_item(dealer, province, city):
	item = {
	"dealer_name" : dealer["name"],
	"brand_id" : None,
	"address" : dealer["address"],
	"brand" : "纳智捷",
	"province" : province,
	"city" : city,
	"sale_call" : dealer["phone_seal"],
	"customer_service_call" : dealer["phone_service"],
	"update_time" : date.today(),
	"longitude" : dealer["lng"],
	"latitude" : dealer["lat"],
	"dealer_type" : None,
	"manufacturer_id" : None,
	"manufacturer" : "东风裕隆",
	"state" : None,
	"opening_date" : None,
	"close_date" : None,
	"dealer_id_web" : None,
	"controlling_shareholder" : None,
	"other_shareholders" : None,
	"status" : None,
	"remarks" : None,
	}
	return item

def get_items():
	items = []

	provinces = get_provinces()
	for province in provinces:
		province_name = province["name"]
		province_id = province["cid"]
		cities = get_cities(province_id)
		for city in cities:
			city_name = city["name"]
			city_id = city["cid"]
			dealers = get_dealers(province_id, city_id)
			for dealer in dealers:
				item = get_item(dealer, province_name, city_name)
				items.append(item)
	return items
def main():
	print("爬虫开始...")
	items = get_items()
	print("数据存库")
	pg = PgsqlPipeline()
	for item in items:
		pg.process_item(item)
	pg.close()
	print("完成")

if __name__ == '__main__':
	main()